const puppeteer = require('puppeteer');
const fs = require('fs');

let totalBlogs = [];

async function main() {
  let totalBlogs = [];

  const tab = 'other';
  const sub_tab = 'cmsblogs';
  const fileName = `cmsblogs.json`;


  const browser = await puppeteer.launch({headless: true, slowMo: 0});
  const page = (await browser.pages())[0];
  await page.setViewport({width: 1500, height: 0});
  page.on('console', msg => {
    console.log(msg.text());
  });

  page.on('pageerror', msg => {
    console.log(msg);
  });


  let catId = 10;
  let pageSize = 2;
  for (let i = 0; i < pageSize; i++) {
    let url = `http://cmsblogs.com/?cat=${catId}&paged=${i + 1}`;
    await page.goto(url);
    await page.waitFor('body > section > div.content-wrap > div > article.excerpt');
    let blogs = await page.$$eval('body > section > div.content-wrap > div > article.excerpt', async (eles) => {
      let tmpBlogs = [];
      eles.forEach(async ele => {
        let title = ele.querySelector('header > h2 > a').innerText;
        let content = ele.querySelector('header > h2 > a').href;
        let blog_date = ele.querySelector('p.meta > time').innerText.replace(/-/g, '/');
        tmpBlogs.push({
          title: title,
          content: content,
          blog_date: blog_date,
          tab: 'other',
          sub_tab: 'cmsblogs'
        });
      });
      return tmpBlogs;
    });
    console.log(blogs);
    console.log('--------------');
    totalBlogs = totalBlogs.concat(blogs);
  }


  let writerStream = fs.createWriteStream(`crawler/crawler_data/${fileName}`);
  writerStream.write(JSON.stringify(totalBlogs));
  console.log('write done');


};


main();


